#!/usr/bin/python2.6
import os

basepath = 'final'

src = basepath+'/final.utf8'
dst = basepath+'/final-oov'

div = basepath+'/div'
oov_list = (('quan', 0x00000080), ('prep', 0x00000040), ('nr_prefix', 0x10000000), ('ns_suffix', 0x20000000), ('nr_right', 0x40000000), ('foreign', 0x80000000))

def output(file, key, freq, tag):
	out = "%s\t0x%x" % (key, freq)
	if(tag > 0): out = out + "\t0x%x" % (tag,)
	file.write(out + "\n")

def on_oov(filepath, tag):
	file = open(filepath, 'r')
	for line in file:
		key = line.strip()
		if(key in hoov):
			val, visited = hoov[key]
		else:
			val = 0
		hoov[key] = (tag|val, 0)
	file.close()

def on_div(filepath):
	file = open(filepath, 'r')
	for line in file:
		arr = line.split()
		if(len(arr) > 3 or len(arr) < 2):
			print "len ", len(arr),": ", line
			continue
		key = arr[0]
		val1 = int(arr[1])*3
		val2 = 0
		if(len(arr) == 3):
			val2 = int(arr[2])*3
		if(val1 > 15 or val2 > 15):
			print "invalid: ", line, val1, val2
			continue
		val = (val1 * (2**4)) + val2
		val = val * (2**24)
		hdiv[key] = (val, 0)
	file.close()

fsrc = open(src, 'r');
fdst = open(dst, 'w');

#key: word;  value: (freq, freq)
hdiv = {}
on_div(div)

#key: word;  value: (tag, freq)
hoov = {}
for filename, tag in oov_list:
	on_oov(basepath+'/'+filename, tag)

for line in fsrc:
	pos = line.find('\t')
	if(pos == -1):
		print "illegal line: " + line
		continue;
	key = line[:pos].strip()
	freq = int(line[pos+1:].strip())
	tag = 0
	if(freq <= 0 or freq >= (2**24)):
		print "illegal freq: " + line
		continue;
	if(key in hdiv):
		hdiv[key] = (hdiv[key][0], 1)
		freq = freq + hdiv[key][0]
	if(key in hoov):
		hoov[key] = (hoov[key][0], 1)
		tag = hoov[key][0]
	output(fdst, key, freq, tag)

for key in hdiv.keys():
	if(hdiv[key][1] == 0):
		freq = 1 + hdiv[key][0]
		output(fdst, key, freq, 0)

for key in hoov.keys():
	if(hoov[key][1] == 0):
		tag = hoov[key][0]
		output(fdst, key, 1, tag)

fsrc.close()
fdst.close()

os.system("mv " + dst + " " + src)
